<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>BERT在情感分析ATSC子任务的应用 | kiloGrand</title><meta name="keywords" content="sentiment analysis"><meta name="author" content="kiloGrand"><meta name="copyright" content="kiloGrand"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="本文主要介绍论文Adapt or Get Left Behind: Domain Adaptation through BERT Language Model Finetuning for Aspect-Target Sentiment Classification及其代码实现。 论文解读Introduction最早的情感分析只是判断文本(可以是一句话或者一段长文本)的情感倾向，但是很多实际应用需">
<meta property="og:type" content="article">
<meta property="og:title" content="BERT在情感分析ATSC子任务的应用">
<meta property="og:url" content="https://kilogrand.gitee.io/2022/10/03/nlp-sentiment-analysis-bert-atsc/index.html">
<meta property="og:site_name" content="kiloGrand">
<meta property="og:description" content="本文主要介绍论文Adapt or Get Left Behind: Domain Adaptation through BERT Language Model Finetuning for Aspect-Target Sentiment Classification及其代码实现。 论文解读Introduction最早的情感分析只是判断文本(可以是一句话或者一段长文本)的情感倾向，但是很多实际应用需">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://kilogrand.gitee.io/img/coding.jpg">
<meta property="article:published_time" content="2022-10-02T16:41:10.000Z">
<meta property="article:modified_time" content="2023-03-17T15:45:16.025Z">
<meta property="article:author" content="kiloGrand">
<meta property="article:tag" content="sentiment analysis">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://kilogrand.gitee.io/img/coding.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://kilogrand.gitee.io/2022/10/03/nlp-sentiment-analysis-bert-atsc/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"/search.xml","preload":false,"languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: 'BERT在情感分析ATSC子任务的应用',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2023-03-17 23:45:16'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.2"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/profile.png" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">46</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">6</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">5</div></a></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> Link</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/img/coding.jpg')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">kiloGrand</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> Link</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">BERT在情感分析ATSC子任务的应用</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2022-10-02T16:41:10.000Z" title="发表于 2022-10-03 00:41:10">2022-10-03</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2023-03-17T15:45:16.025Z" title="更新于 2023-03-17 23:45:16">2023-03-17</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/NLP/">NLP</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">2.5k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>10分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="BERT在情感分析ATSC子任务的应用"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><p>本文主要介绍论文<a target="_blank" rel="noopener" href="https://arxiv.org/abs/1908.11860">Adapt or Get Left Behind: Domain Adaptation through BERT Language Model Finetuning for Aspect-Target Sentiment Classification</a>及其代码实现。</p>
<h2 id="论文解读"><a href="#论文解读" class="headerlink" title="论文解读"></a>论文解读</h2><h3 id="Introduction"><a href="#Introduction" class="headerlink" title="Introduction"></a>Introduction</h3><p>最早的情感分析只是判断文本(可以是一句话或者一段长文本)的情感倾向，但是很多实际应用需要更细粒度的分析，这就出现了Aspect Based Sentiment Analysis (ABSA)任务，不了解的读者可以先参考<a href="/2019/09/25/sentiment-analysis-survey/">情感分析简介</a>。ABSA任务有两种方法：ACD+ACSC和ATE+ATSC。</p>
<p>ACD是Aspect Category Detection的缩写，ACSC是Aspect Category Sentiment Classification的缩写。ACD是一个多标签(multi-label)分类问题，一个句子可以同时说多个aspect category。以句子”I love their dumplings”为例，ACD会把它分类为food这个category，而ACSC会把这个aspect的情感分类为正面。</p>
<p>ATE是Aspect Target Extraction的缩写，而ATSC是Aspect Target Sentiment Classification的缩写。还是以”I love their dumplings”为例，ATE抽取的是dumplings，ATSC会把对于dumplings的情感分类为正面。</p>
<p>本文解决的就是ATSC的问题，也就是给定一个句子和Aspect Target(比如dumplings)，判断它的情感分类。注意：一个句子可能包括多个Aspect Target，比如”这个酒店的位置很好但是服务一般”，则它有”位置”和”服务”两个Aspect Target，它们的情感分类分别是正面和负面。对于这个句子，会进行两次预测，首先的输入是”这个酒店的位置很好但是服务一般”+”位置”，输出应该是正面；接着输入是”这个酒店的位置很好但是服务一般”+”服务”，输出是负面。</p>
<h3 id="方法"><a href="#方法" class="headerlink" title="方法"></a>方法</h3><p>本文的方法非常简单，就是使用BERT来进行分类，对BERT不熟悉的读者可以先参考<a href="/2019/03/05/bert-prerequisites/">BERT课程</a>。因为有句子和Target两个输入，所以在fine-tuning是会把它们拼接起来。假设句子是s，Target是t，则BERT在fine-tuning时的输入是”[CLS] s [SEP] t [SEP]”。另外本文能取得很好结果的原因就是使用了大量领域数据来pretraining BERT，因为Wiki等语料库和评论的差别还是比较大的。比如在wiki里，”The touchscreen is an [MASK] device”，[MASK]很可能是”input”这样的词，而在评论里，[MASK]更可能是”amazing”这样的词。</p>
<p>论文使用了Yelp的酒店评论数据和Amazon的电子产品的评论数据来对BERT模型进行pretraining(初始为Google模型)，在酒店的ATSC任务上取得里很好的成绩，下图是实验结果。</p>
<p><a name='img1'><img src="/img/nlp-sentiment-analysis-bert-atsc.png" alt=""></a><br><em>图：实验结果</em></p>
<p>这篇文章的方法在酒店的数据集上效果很好，但是在笔记本电脑上并没有取得最好的结果。</p>
<h2 id="代码"><a href="#代码" class="headerlink" title="代码"></a>代码</h2><h3 id="运行代码"><a href="#运行代码" class="headerlink" title="运行代码"></a>运行代码</h3><h4 id="下载代码"><a href="#下载代码" class="headerlink" title="下载代码"></a>下载代码</h4><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">git clone https://github.com/deepopinion/domain-adapted-atsc.git</span><br></pre></td></tr></table></figure>
<h4 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h4><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">python -m venv venv</span><br><span class="line">source venv/bin/activate</span><br><span class="line">pip install -r requirements.txt</span><br><span class="line">python -m spacy download en_core_web_sm</span><br><span class="line">mkdir -p data/raw/semeval2014  # creates directories for data</span><br><span class="line">mkdir -p data/transformed</span><br><span class="line">mkdir -p data/models</span><br></pre></td></tr></table></figure>
<p>为了进行fine-tuning，需要安装pytorch、pytorch-transformers和apex。我们首先安装pytorch和pytorch-transformer：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">pip install scipy sckit-learn</span><br><span class="line">pip install https://download.pytorch.org/whl/cu100/torch-1.1.0-cp36-cp36m-linux_x86_64.whl</span><br><span class="line">pip install pytorch-transformers tensorboardX</span><br></pre></td></tr></table></figure>
<p>注意：上面安装的是pytorch-1.1.0的GPU版本，它需要CUDA-10.0。</p>
<p>接着需要安装apex：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">cd ..</span><br><span class="line">git clone https://github.com/NVIDIA/apex</span><br><span class="line">cd apex</span><br><span class="line">pip install -v --no-cache-dir --global-option=&quot;--cpp_ext&quot; --global-option=&quot;--cuda_ext&quot; ./</span><br></pre></td></tr></table></figure>
<p>作者执行最后的pip install时碰到了一些小困难。它会提示nvcc的版本和编译pytorch的不一致(因为pytorch是用pip而不是从源代码安装的)，因此需要修改setup.py去掉下面的检查的代码：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">check_cuda_torch_binary_vs_bare_metal(torch.utils.cpp_extension.CUDA_HOME)</span><br></pre></td></tr></table></figure>
<p>去掉之后就可以用最后pip install 安装apex了。</p>
<h4 id="准备fine-tuning-BERT语言模型的数据"><a href="#准备fine-tuning-BERT语言模型的数据" class="headerlink" title="准备fine-tuning BERT语言模型的数据"></a>准备fine-tuning BERT语言模型的数据</h4><p>这个步骤是准备fine-tuning(其实是在BERT基础模型的基础上继续pretraining)语言模型的数据，作者也提供了他fine-tuning之后的模型，如果读者不想自己fine-tuning语言模型可以跳过这一步。</p>
<p>作者用来fine-tuning laptop任务的数据来自Amazon的电子产品的数据，参考<a target="_blank" rel="noopener" href="http://jmcauley.ucsd.edu/data/amazon/amazon_readme.txt">这个链接</a>，大家可以发邮件给julian.mcauley@gmail.com来申请这个数据集。mcauley的邮件会给出下载的链接，读者也需要下载的是reviews_Electronics.json.gz和meta_Electronics.json.gz两个文件，注意别下载错了。这两个文件分别为1.7GB和178MB。</p>
<p>而fine-tuning restaurant的数据集来自yelp，大家可以点击<a target="_blank" rel="noopener" href="https://www.yelp.com/dataset/download">这里</a>下载，下载得到一个yelp_dataset.tar.gz。解压它可以得到一个review.json文件，这个文件的大小是5.0GB。</p>
<p>把这些文件都放到data/raw下，类似：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">lili@lili-Precision-7720:~/codes/domain-adapted-atsc/data/raw$ ls</span><br><span class="line">meta_Electronics.json.gz  review.json  reviews_Electronics.json.gz</span><br></pre></td></tr></table></figure>
<p>数据预处理：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">python prepare_laptop_reviews.py</span><br><span class="line">python prepare_restaurant_reviews.py</span><br><span class="line">python prepare_restaurant_reviews.py --large  # takes some time to finish</span><br></pre></td></tr></table></figure>
<p>处理后在data/transformed/下会得到laptop_corpus_1019917.txt这样的文件，这是BERT pretraining需要的数据格式，我们可以看几行：<br><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">$ head laptop_corpus_1019917.txt </span><br><span class="line">This product has proven to be a communication breakthrough for my brother who has gone deaf in his elder years.</span><br><span class="line">He is not computer literate and cannot type.</span><br><span class="line">However, he quickly picked up on how to use this machine which plugs into the phone line.</span><br><span class="line">I love it!</span><br><span class="line">It has ended our one way conversations for he can now read whatever I send him and either respond by email or by calling (I can hear him just fine).</span><br><span class="line">I think this is a very useful product!</span><br><span class="line">I gave this product 1 star based on problems I have encountered with one I bought for my in-laws.</span><br><span class="line">(See previous review).</span><br></pre></td></tr></table></figure></p>
<p>文档之间用一个空行分割开，每行表示一个句子，这是BERT需要的。</p>
<p>因为论文还把两个数据集混合在一起训练，所以还有一个步骤是把两个数据cat到一起：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">cd data/transformed</span><br><span class="line">cat laptop_corpus_1011255.txt restaurant_corpus_1000004.txt &gt; mixed_corpus.txt</span><br></pre></td></tr></table></figure>
<h4 id="下载SemEval-2014数据"><a href="#下载SemEval-2014数据" class="headerlink" title="下载SemEval 2014数据"></a>下载SemEval 2014数据</h4><p>请读者去<a target="_blank" rel="noopener" href="http://metashare.ilsp.gr:8080/repository/search/?q=semeval+2014">这里</a>所有SemEval 2014的数据，下载后类似这样：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">lili@lili-Precision-7720:~/codes/domain-adapted-atsc/data/raw/semeval2014$ tree</span><br><span class="line">.</span><br><span class="line">├── SemEval-2014 ABSA Test Data - Gold Annotations</span><br><span class="line">│   ├── ABSA_Gold_TestData</span><br><span class="line">│   │   ├── Laptops_Test_Gold.xml</span><br><span class="line">│   │   └── Restaurants_Test_Gold.xml</span><br><span class="line">│   └── Laptops_Test_Gold.xml</span><br><span class="line">└── SemEval-2014 ABSA Train Data v2.0 &amp; Annotation Guidelines</span><br><span class="line">    ├── Laptop_Train_v2.xml</span><br><span class="line">    └── Restaurants_Train_v2.xml</span><br></pre></td></tr></table></figure>
<p>请参考上面的目录结构放置解压后的文件(如果是Windows的话可能文件名不能用&amp;，那么需要重命名，代码也需要修改)。</p>
<p>我们看一下SemEval-2014 ABSA Train Data v2.0 &amp; Annotation Guidelines/Restaurants_Train_v2.xml这个文件：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"> 1 &lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;yes&quot;?&gt;</span><br><span class="line"> 2 &lt;sentences&gt;</span><br><span class="line"> 3     &lt;sentence id=&quot;3121&quot;&gt;</span><br><span class="line"> 4         &lt;text&gt;But the staff was so horrible to us.&lt;/text&gt;</span><br><span class="line"> 5         &lt;aspectTerms&gt;</span><br><span class="line"> 6             &lt;aspectTerm term=&quot;staff&quot; polarity=&quot;negative&quot; from=&quot;8&quot; to=&quot;13&quot;/&gt;</span><br><span class="line"> 7         &lt;/aspectTerms&gt;</span><br><span class="line"> 8         &lt;aspectCategories&gt;</span><br><span class="line"> 9             &lt;aspectCategory category=&quot;service&quot; polarity=&quot;negative&quot;/&gt;</span><br><span class="line">10         &lt;/aspectCategories&gt;</span><br><span class="line">11     &lt;/sentence&gt;</span><br><span class="line">12     &lt;sentence id=&quot;2777&quot;&gt;</span><br><span class="line">13         &lt;text&gt;To be completely fair, the only redeeming factor was the food, which was above average, but co      uldn&#x27;t make up for all the other deficiencies of Teodora.&lt;/text&gt;</span><br><span class="line">14         &lt;aspectTerms&gt;</span><br><span class="line">15             &lt;aspectTerm term=&quot;food&quot; polarity=&quot;positive&quot; from=&quot;57&quot; to=&quot;61&quot;/&gt;</span><br><span class="line">16         &lt;/aspectTerms&gt;</span><br><span class="line">17         &lt;aspectCategories&gt;</span><br><span class="line">18             &lt;aspectCategory category=&quot;food&quot; polarity=&quot;positive&quot;/&gt;</span><br><span class="line">19             &lt;aspectCategory category=&quot;anecdotes/miscellaneous&quot; polarity=&quot;negative&quot;/&gt;</span><br><span class="line">20         &lt;/aspectCategories&gt;</span><br><span class="line">21     &lt;/sentence&gt;</span><br></pre></td></tr></table></figure>
<p>对于ATSC这个任务，输入是”But the staff was so horrible to us.”和”staff”，输出是negative这个分类。</p>
<p>因为SemEval 2014分类包括冲突(conflict)，作者把冲突的数据去掉了。下面的脚本就是处理掉这些数据，首先是laptop的数据：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"># laptops</span><br><span class="line"># laptops without conflict label</span><br><span class="line">python prepare_semeval_datasets.py \</span><br><span class="line">--files &quot;data/raw/semeval2014/SemEval-2014 ABSA Train Data v2.0 &amp; Annotation Guidelines/Laptop_Train_v2.xml&quot; \</span><br><span class="line">--output_dir data/transformed/laptops_noconfl \</span><br><span class="line">--istrain \</span><br><span class="line">--noconfl</span><br><span class="line">python prepare_semeval_datasets.py \</span><br><span class="line">--files &quot;data/raw/semeval2014/SemEval-2014 ABSA Test Data - Gold Annotations/ABSA_Gold_TestData/Laptops_Test_Gold.xml&quot; \</span><br><span class="line">--output_dir data/transformed/laptops_noconfl \</span><br><span class="line">--noconfl</span><br></pre></td></tr></table></figure>
<p>然后是restaurant：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"># restaurants without conflict label</span><br><span class="line">python prepare_semeval_datasets.py \</span><br><span class="line">--files &quot;data/raw/semeval2014/SemEval-2014 ABSA Train Data v2.0 &amp; Annotation Guidelines/Restaurants_Train_v2.xml&quot; \</span><br><span class="line">--output_dir data/transformed/restaurants_noconfl \</span><br><span class="line">--istrain \</span><br><span class="line">--noconfl</span><br><span class="line">python prepare_semeval_datasets.py \</span><br><span class="line">--files &quot;data/raw/semeval2014/SemEval-2014 ABSA Test Data - Gold Annotations/ABSA_Gold_TestData/Restaurants_Test_Gold.xml&quot; \</span><br><span class="line">--output_dir data/transformed/restaurants_noconfl \</span><br><span class="line">--noconfl</span><br></pre></td></tr></table></figure>
<p>最后是混合的训练数据：<br><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"># mixed without conflict label</span><br><span class="line">python prepare_semeval_datasets.py \</span><br><span class="line">--files &quot;data/raw/semeval2014/SemEval-2014 ABSA Train Data v2.0 &amp; Annotation Guidelines/Restaurants_Train_v2.xml&quot; \</span><br><span class="line">&quot;data/raw/semeval2014/SemEval-2014 ABSA Train Data v2.0 &amp; Annotation Guidelines/Laptop_Train_v2.xml&quot; \</span><br><span class="line">--output_dir data/transformed/mixed_noconfl \</span><br><span class="line">--istrain --noconfl</span><br><span class="line">python prepare_semeval_datasets.py \</span><br><span class="line">--files &quot;data/raw/semeval2014/SemEval-2014 ABSA Test Data - Gold Annotations/ABSA_Gold_TestData/Restaurants_Test_Gold.xml&quot; \</span><br><span class="line">&quot;data/raw/semeval2014/SemEval-2014 ABSA Test Data - Gold Annotations/ABSA_Gold_TestData/Laptops_Test_Gold.xml&quot; \</span><br><span class="line">--output_dir data/transformed/mixed_noconfl --noconfl</span><br></pre></td></tr></table></figure></p>
<h4 id="使用作者pretraining好的BERT语言模型来fine-tuning-ATSC-restaurant任务"><a href="#使用作者pretraining好的BERT语言模型来fine-tuning-ATSC-restaurant任务" class="headerlink" title="使用作者pretraining好的BERT语言模型来fine-tuning ATSC restaurant任务"></a>使用作者pretraining好的BERT语言模型来fine-tuning ATSC restaurant任务</h4><p>我们这里只尝试restaurant数据集，首先在<a target="_blank" rel="noopener" href="https://drive.google.com/file/d/1DmVrhKQx74p1U5c7oq6qCTVxGIpgvp1c/view?usp=sharing">这里</a>下载作者pretraining好的BERT模型。下载后放到data/models下：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">lili@lili-Precision-7720:~/codes/domain-adapted-atsc/data/models$ tree restaurants_10mio_ep3/</span><br><span class="line">restaurants_10mio_ep3/</span><br><span class="line">├── added_tokens.json</span><br><span class="line">├── config.json</span><br><span class="line">├── pytorch_model.bin</span><br><span class="line">├── special_tokens_map.json</span><br><span class="line">└── vocab.txt</span><br></pre></td></tr></table></figure>
<p>然后基于这个模型进行监督的fine-tuning：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">cd finetuning_and_classification</span><br><span class="line">python run_glue.py \ </span><br><span class="line">--model_type bert \</span><br><span class="line">--model_name_or_path ../data/models/restaurants_10mio_ep3 \</span><br><span class="line">--do_train --evaluate_during_training --do_eval \</span><br><span class="line">--logging_steps 100 --save_steps 1200 --task_name=semeval2014-atsc \</span><br><span class="line">--seed 42 --do_lower_case \</span><br><span class="line">--data_dir=../data/transformed/restaurants_noconfl \</span><br><span class="line">--output_dir=../data/models/semeval2014-atsc-bert-ada-restaurants-restaurants \</span><br><span class="line">--max_seq_length=128 --learning_rate 3e-5 --per_gpu_eval_batch_size=8 --per_gpu_train_batch_size=8 \</span><br><span class="line">--gradient_accumulation_steps=1 --max_steps=800 --overwrite_output_dir --overwrite_cache --warmup_steps=120 --fp16</span><br></pre></td></tr></table></figure>
<p>请根据GPU的内存修改per_gpu_eval_batch_size和per_gpu_train_batch_size两个参数，我这里使用是8。</p>
<p>最终作者得到的结果为：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">10/21/2019 09:41:51 - INFO - __main__ -   ***** Eval results  *****</span><br><span class="line">10/21/2019 09:41:51 - INFO - __main__ -     acc = 0.8723214285714286</span><br><span class="line">10/21/2019 09:41:51 - INFO - __main__ -     f1_macro = 0.7945154951637271</span><br></pre></td></tr></table></figure>
<p>基本和论文里的87%的准确率以及80%的F1得分是差不多的。</p>
<h4 id="自己使用Yelp来pretrainng语言模型"><a href="#自己使用Yelp来pretrainng语言模型" class="headerlink" title="自己使用Yelp来pretrainng语言模型"></a>自己使用Yelp来pretrainng语言模型</h4><p>因为训练1000万的语料太费时间，我这里只使用里100万的数据进行了3个epoch。</p>
<p>首先需要生成BERT需要的训练数据，这可以使用下面的脚本：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">python pregenerate_training_data.py \</span><br><span class="line">--train_corpus \</span><br><span class="line">../data/transformed/restaurant_corpus_1000000.txt \</span><br><span class="line">--bert_model \</span><br><span class="line">bert-base-uncased \</span><br><span class="line">--do_lower_case \</span><br><span class="line">--output_dir \</span><br><span class="line">dev_corpus_prepared/ \</span><br><span class="line">--epochs_to_generate \</span><br><span class="line">3 \</span><br><span class="line">--max_seq_len \</span><br><span class="line">256</span><br></pre></td></tr></table></figure>
<p>接着进行pretraining：<br><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">python finetune_on_pregenerated.py \</span><br><span class="line">--pregenerated_data dev_corpus_prepared/ \</span><br><span class="line">--bert_model bert-base-uncased \</span><br><span class="line">--do_lower_case \</span><br><span class="line">--output_dir dev_corpus_finetuned/ \</span><br><span class="line">--epochs 2 \</span><br><span class="line">--train_batch_size 4 \</span><br></pre></td></tr></table></figure></p>
<p>然后用自己pretraining的模型再进行fine-tuning：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">python run_glue.py \ </span><br><span class="line">--model_type bert \</span><br><span class="line">--model_name_or_path dev_corpus_finetuned/ \</span><br><span class="line">--do_train --evaluate_during_training --do_eval \</span><br><span class="line">--logging_steps 100 --save_steps 1200 --task_name=semeval2014-atsc \</span><br><span class="line">--seed 42 --do_lower_case \</span><br><span class="line">--data_dir=../data/transformed/restaurants_noconfl \</span><br><span class="line">--output_dir=../data/models/semeval2014-atsc-bert-ada-restaurants-restaurants \</span><br><span class="line">--max_seq_length=128 --learning_rate 3e-5 --per_gpu_eval_batch_size=8 --per_gpu_train_batch_size=8 \</span><br><span class="line">--gradient_accumulation_steps=1 --max_steps=800 --overwrite_output_dir --overwrite_cache --warmup_steps=120 --fp16</span><br></pre></td></tr></table></figure>
<p>这个和前面的唯一区别就是--model_name_or_path使用了我们自己的模型。因为没有使用大量的数据，最终的效果比作者pretraining的要差一些。从这里也能看出，如果读者要对自己领域的数据进行情感分析的话，最好还是找大量未标注的语料库pretraining之后在用监督数据进行fine-tuning效果会更好。</p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="https://kilogrand.gitee.io">kiloGrand</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://kilogrand.gitee.io/2022/10/03/nlp-sentiment-analysis-bert-atsc/">https://kilogrand.gitee.io/2022/10/03/nlp-sentiment-analysis-bert-atsc/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://kilogrand.gitee.io" target="_blank">kiloGrand</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/sentiment-analysis/">sentiment analysis</a></div><div class="post_share"></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/2022/10/02/nlp-sentiment-analysis-survey/"><img class="prev-cover" src="/img/coding.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">情感分析简介</div></div></a></div><div class="next-post pull-right"><a href="/2023/02/26/tuning_playbook-zh_cn/"><img class="next-cover" src="/img/coding.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">深度学习调优指南中文版</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/2022/10/01/nlp-sentiment-analysis-dataset/" title="情感分析常见数据集介绍"><img class="cover" src="/img/coding.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-10-01</div><div class="title">情感分析常见数据集介绍</div></div></a></div><div><a href="/2022/10/02/nlp-sentiment-analysis-survey/" title="情感分析简介"><img class="cover" src="/img/coding.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-10-02</div><div class="title">情感分析简介</div></div></a></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/img/profile.png" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">kiloGrand</div><div class="author-info__description">coder && data-science researcher</div></div><div class="card-info-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">46</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">6</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">5</div></a></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/kiloGrand/"><i class="fab fa-github"></i><span>Follow Me</span></a></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">This is my Blog</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%AE%BA%E6%96%87%E8%A7%A3%E8%AF%BB"><span class="toc-number">1.</span> <span class="toc-text">论文解读</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Introduction"><span class="toc-number">1.1.</span> <span class="toc-text">Introduction</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%96%B9%E6%B3%95"><span class="toc-number">1.2.</span> <span class="toc-text">方法</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BB%A3%E7%A0%81"><span class="toc-number">2.</span> <span class="toc-text">代码</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%BF%90%E8%A1%8C%E4%BB%A3%E7%A0%81"><span class="toc-number">2.1.</span> <span class="toc-text">运行代码</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%B8%8B%E8%BD%BD%E4%BB%A3%E7%A0%81"><span class="toc-number">2.1.1.</span> <span class="toc-text">下载代码</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%AE%89%E8%A3%85"><span class="toc-number">2.1.2.</span> <span class="toc-text">安装</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%87%86%E5%A4%87fine-tuning-BERT%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%E7%9A%84%E6%95%B0%E6%8D%AE"><span class="toc-number">2.1.3.</span> <span class="toc-text">准备fine-tuning BERT语言模型的数据</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%B8%8B%E8%BD%BDSemEval-2014%E6%95%B0%E6%8D%AE"><span class="toc-number">2.1.4.</span> <span class="toc-text">下载SemEval 2014数据</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BD%BF%E7%94%A8%E4%BD%9C%E8%80%85pretraining%E5%A5%BD%E7%9A%84BERT%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%E6%9D%A5fine-tuning-ATSC-restaurant%E4%BB%BB%E5%8A%A1"><span class="toc-number">2.1.5.</span> <span class="toc-text">使用作者pretraining好的BERT语言模型来fine-tuning ATSC restaurant任务</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E8%87%AA%E5%B7%B1%E4%BD%BF%E7%94%A8Yelp%E6%9D%A5pretrainng%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B"><span class="toc-number">2.1.6.</span> <span class="toc-text">自己使用Yelp来pretrainng语言模型</span></a></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/2023/03/24/kuiper_infer-L14/" title="自制深度学习框架--实现Yolov5的推理">自制深度学习框架--实现Yolov5的推理</a><time datetime="2023-03-24T12:00:00.000Z" title="发表于 2023-03-24 20:00:00">2023-03-24</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/2023/03/23/kuiper_infer-L13/" title="自制深度学习框架--实现ResNet网络的推理">自制深度学习框架--实现ResNet网络的推理</a><time datetime="2023-03-23T12:00:00.000Z" title="发表于 2023-03-23 20:00:00">2023-03-23</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/2023/03/22/kuiper_infer-L12/" title="自制深度学习框架--算子的执行流程">自制深度学习框架--算子的执行流程</a><time datetime="2023-03-22T12:00:00.000Z" title="发表于 2023-03-22 20:00:00">2023-03-22</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/2023/03/21/kuiper_infer-L11/" title="自制深度学习框架--再探Tensor类并构建计算图的图关系">自制深度学习框架--再探Tensor类并构建计算图的图关系</a><time datetime="2023-03-21T12:00:00.000Z" title="发表于 2023-03-21 20:00:00">2023-03-21</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/2023/03/20/kuiper_infer-L10/" title="自制深度学习框架--Im2Col原理与卷积层的实现">自制深度学习框架--Im2Col原理与卷积层的实现</a><time datetime="2023-03-20T12:00:00.000Z" title="发表于 2023-03-20 20:00:00">2023-03-20</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2022 - 2023 By kiloGrand</div><div class="footer_custom_text">Hi, welcome to my blog!</div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">搜索</span><span id="loading-status"></span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="is-center" id="loading-database"><i class="fas fa-spinner fa-pulse"></i><span>  数据库加载中</span></div><div class="search-wrap"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div><hr/><div id="local-search-results"></div></div></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.js"></script><script src="/js/search/local-search.js"></script><div class="js-pjax"><script>if (!window.MathJax) {
  window.MathJax = {
    tex: {
      inlineMath: [ ['$','$'], ["\\(","\\)"]],
      tags: 'ams'
    },
    chtml: {
      scale: 1.2
    },
    options: {
      renderActions: {
        findScript: [10, doc => {
          for (const node of document.querySelectorAll('script[type^="math/tex"]')) {
            const display = !!node.type.match(/; *mode=display/)
            const math = new doc.options.MathItem(node.textContent, doc.inputJax[0], display)
            const text = document.createTextNode('')
            node.parentNode.replaceChild(text, node)
            math.start = {node: text, delim: '', n: 0}
            math.end = {node: text, delim: '', n: 0}
            doc.math.push(math)
          }
        }, ''],
        insertScript: [200, () => {
          document.querySelectorAll('mjx-container:not\([display]\)').forEach(node => {
            const target = node.parentNode
            if (target.nodeName.toLowerCase() === 'li') {
              target.parentNode.classList.add('has-jax')
            } else {
              target.classList.add('has-jax')
            }
          });
        }, '', false]
      }
    }
  }
  
  const script = document.createElement('script')
  script.src = 'https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js'
  script.id = 'MathJax-script'
  script.async = true
  document.head.appendChild(script)
} else {
  MathJax.startup.document.state(0)
  MathJax.texReset()
  MathJax.typeset()
}</script></div><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>